﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Diagnostics;
using System.Text.RegularExpressions;
using System.Net;
using System.Threading;
using System.IO;

namespace CNSegmenter
{
    public class RequestState
    {
        // This class stores the state of the request.
        public int BUFFER_SIZE = 1024;
        public StringBuilder requestData;
        public byte[] bufferRead;
        public WebRequest request;
        public WebResponse response;
        public Stream responseStream;
        public string url;
        public string filePath;
        public ManualResetEvent allDone = null;
        public RequestState()
        {
            bufferRead = new byte[BUFFER_SIZE];
            requestData = new StringBuilder("");
            request = null;
            responseStream = null;
            url = "";
            filePath = "";
            allDone = new ManualResetEvent(false);
        }
    }

    class Spider
    {
        public ManualResetEvent manualEvent;
        private int _maxVisitedURLs = 1000;
        public int MaxVisitedURLs
        {
            get
            {
                return _maxVisitedURLs;
            }
            set
            {
                _maxVisitedURLs = value;
            }
        }

        public void AddURL(string url)
        {
            lock (_unvisitedURLs)
            {
                _unvisitedURLs.Enqueue(url);
            }
        }

        private Queue<string> _unvisitedURLs = new Queue<string>();
        private HashSet<string> _visitiedURLs = new HashSet<string>();
        static public int MaxDownloadFilesCount = 50;
        public int start = 0;
        public int end = 0;
        public void RunSpider()
        {
            while (true)
            {
                //if (_fileCount >= MaxDownloadFilesCount)
                //{
                //    break;
                //}
                if (start >= end)
                {
                    manualEvent.Set();
                    Console.WriteLine("====" + start.ToString() + "finish!" + "===");
                    break;
                }

                //if (_unvisitedURLs.Count > 0)
                //{
                    RequestState requestState = new RequestState();
                    //lock (_unvisitedURLs)
                    //{
                    //    requestState.url  = _unvisitedURLs.Dequeue();
                    //}
                    requestState.url = string.Format("http://www.xinli001.com/info/{0}/", start++);

                    lock (_visitiedURLs)
                    {
                        _visitiedURLs.Add(requestState.url);
                    }

                    try
                    {
                        // Create a new webrequest to the mentioned URL.   
                        WebRequest myWebRequest = WebRequest.Create(requestState.url);

                        // The 'WebRequest' object is associated to the 'RequestState' object.
                        requestState.request = myWebRequest;

                        // Start the Asynchronous call for response.
                        IAsyncResult asyncResult = (IAsyncResult)myWebRequest.BeginGetResponse(new AsyncCallback(RespCallback), requestState);
                        requestState.allDone.WaitOne();

                        Debug.Assert(requestState.response != null);
                        // Release the WebResponse resource.
                        requestState.response.Close();
                    }
                    catch (WebException e)
                    {
                        Console.WriteLine("WebException raised! 1");
                        Console.WriteLine("\n{0}", e.Message);
                        Console.WriteLine("\n{0}", e.Status);
                        requestState.request.Abort();
                    }
                    catch (Exception e)
                    {
                        Console.WriteLine("Exception raised!");
                        Console.WriteLine("Source : " + e.Source);
                        Console.WriteLine("Message : " + e.Message);
                    }
                //}
                //else
                //{
                //    // Wait for a sec for new urls.
                //    Console.WriteLine(_unvisitedURLs.Count.ToString() + ": wait for a sec.");
                //    Thread.Sleep(1000);
                //}
            }
        }

        private void RespCallback(IAsyncResult asynchronousResult)
        {
            try
            {
                // Set the State of request to asynchronous.
                RequestState myRequestState = (RequestState)asynchronousResult.AsyncState;
                WebRequest myWebRequest1 = myRequestState.request;
                // End the Asynchronous response.
                myRequestState.response = myWebRequest1.EndGetResponse(asynchronousResult);
                // Read the response into a 'Stream' object.
                Stream responseStream = myRequestState.response.GetResponseStream();
                myRequestState.responseStream = responseStream;
                // Begin the reading of the contents of the HTML page and print it to the console.
                IAsyncResult asynchronousResultRead = responseStream.BeginRead(
                    myRequestState.bufferRead,
                    0,
                    myRequestState.BUFFER_SIZE,
                    new AsyncCallback(ReadCallBack),
                    myRequestState);

            }
            catch (WebException e)
            {
                Console.WriteLine("WebException raised! 2");
                Console.WriteLine("\n{0}", e.Message);
                Console.WriteLine("\n{0}", e.Status);
                RunSpider();
            }
            catch (Exception e)
            {
                Console.WriteLine("Exception raised!");
                Console.WriteLine("Source : " + e.Source);
                Console.WriteLine("Message : " + e.Message);
            }
        }

        static private int _fileCount = 0;
        private void ReadCallBack(IAsyncResult asyncResult)
        {
            try
            {
                // Result state is set to AsyncState.
                RequestState myRequestState = (RequestState)asyncResult.AsyncState;
                Stream responseStream = myRequestState.responseStream;
                int read = responseStream.EndRead(asyncResult);

                // Read the contents of the HTML page and then print to the console.
                myRequestState.requestData.Append(Encoding.UTF8.GetString(myRequestState.bufferRead, 0, read));
                if (read > 0)
                {
                    IAsyncResult asynchronousResult = responseStream.BeginRead(
                        myRequestState.bufferRead, 
                        0, 
                        myRequestState.BUFFER_SIZE, 
                        new AsyncCallback(ReadCallBack), 
                        myRequestState);
                }
                else
                {
                    if (myRequestState.requestData.Length > 1)
                    {
                        string stringContent;
                        stringContent = myRequestState.requestData.ToString();

                        // Save content to file.
                        Match contentMatch = Regex.Match(
                            myRequestState.url,
                            "info/\\d+/", 
                            RegexOptions.Multiline);
                        if (contentMatch.Success)
                        {
                            lock (_visitiedURLs)
                            {
                                myRequestState.filePath = string.Format(@".\Data\{0}.txt", start);  /////////
                            }
                            File.WriteAllText(myRequestState.filePath, stringContent, Encoding.Default);
                            _fileCount++;
                        }
                        

                        // Enqueue all href from content.
                        MatchCollection matches = Regex.Matches(
                            stringContent, 
                            "<a href=\"(?<url>http://[^\"]+)\">", RegexOptions.Multiline);
                        lock (_unvisitedURLs)
                        {
                            foreach (Match match in matches)
                            {
                                _unvisitedURLs.Enqueue(match.Groups["url"].Value);
                                Debug.Print(match.Groups["url"].Value);
                            }
                        }
                        Console.WriteLine("[done] " + myRequestState.url);
                    }
                    responseStream.Close();
                    myRequestState.allDone.Set();
                }
            }
            catch (WebException e)
            {
                Console.WriteLine("WebException raised! 3");
                Console.WriteLine("\n{0}", e.Message);
                Console.WriteLine("\n{0}", e.Status);
            }
            catch (Exception e)
            {
                Console.WriteLine("Exception raised!");
                Console.WriteLine("Source : {0}", e.Source);
                Console.WriteLine("Message : {0}", e.Message);
            }

        }

    }
}
